{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 这是一个时间滑窗函数，获得dt之前minus天以来periods的dataframe，以便进一步计算\n",
    "def get_timespan(df, dt, minus, periods, freq='D'):\n",
    "    return df[pd.date_range(dt - timedelta(days=minus), periods=periods, freq=freq)]\n",
    "\n",
    "# 要计算统计指标特征的时间窗口\n",
    "# for i in [14,30,60,91]:\n",
    "#     tmp = get_timespan(df_payment, t2018, i, i)\n",
    "#    # 削去峰值的均值特征\n",
    "#     X['mean_%s_decay' % i] = (tmp * np.power(0.9, np.arange(i)[::-1])).sum(axis=1).values\n",
    "#    # 中位数特征，在本赛题中基本不适用\n",
    "#    # X['median_%s' % i] = tmp.median(axis=1).values\n",
    "#    # 最小值特征，在本赛题中基本不适用\n",
    "#    # X['min_%s' % i] = tmp_1.min(axis=1).values\n",
    "#    # 最大值特征\n",
    "#     X['max_%s' % i] = tmp.max(axis=1).values\n",
    "#    # 标准差特征\n",
    "#    # X['std_%s' % i] = tmp_1.std(axis=1).values\n",
    "#    # 求和特征\n",
    "#     X['sum_%s' % i] = tmp.sum(axis=1).values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 构建多层神经网络\n",
    "# class Regressor(fluid.dygraph.Layer):\n",
    "#     def __init__(self, name_scope):\n",
    "#         super(Regressor, self).__init__(name_scope)\n",
    "#         name_scope = self.full_name()\n",
    "#         # 定义三层全连接层，输入维度是最终选取的特征数量，输出维度是1，激活函数为relu\n",
    "#         self.fc1 = Linear(input_dim=41, output_dim=128, act='relu') # 输入层，input dim 为数据维度大小\n",
    "#         self.fc2 = Linear(input_dim=128, output_dim=128, act='relu')\n",
    "#         self.fc3 = Linear(input_dim=128, output_dim=1, act='sigmoid')\n",
    "#     # 网络的前向计算函数\n",
    "#     def forward(self, inputs):\n",
    "#         fc1 = self.fc1(inputs)\n",
    "#         fc2 = self.fc2(fc1)\n",
    "#         x = self.fc3(fc2)\n",
    "#         return x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 数据预处理 - 数据集划分与特征工程"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np  #\n",
    "import pandas as pd  #\n",
    "from datetime import datetime, date, timedelta\n",
    "from scipy.stats import skew  # for some statistics\n",
    "from scipy.special import boxcox1p\n",
    "from scipy.stats import boxcox_normmax\n",
    "# from sklearn.linear_model import ElasticNetCV, LassoCV, RidgeCV,Ridge,Lasso,ElasticNet\n",
    "# from sklearn.ensemble import GradientBoostingRegressor, RandomForestRegressor,RandomForestClassifier\n",
    "# from sklearn.feature_selection import mutual_info_regression\n",
    "# from sklearn.svm import SVR, LinearSVC\n",
    "# from sklearn.pipeline import make_pipeline,Pipeline\n",
    "# from sklearn.preprocessing import RobustScaler, LabelEncoder, StandardScaler,MinMaxScaler\n",
    "# from sklearn.model_selection import KFold, cross_val_score, train_test_split\n",
    "# from sklearn.metrics import mean_squared_error\n",
    "# from sklearn.model_selection import GridSearchCV\n",
    "# from sklearn.model_selection import train_test_split\n",
    "import os\n",
    "import re\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import time\n",
    "from itertools import product\n",
    "import datetime as dt\n",
    "import calendar\n",
    "import gc\n",
    "from datetime import date, timedelta\n",
    "import paddle\n",
    "import paddle.fluid as fluid\n",
    "import paddle.fluid.dygraph as dygraph\n",
    "from paddle.fluid.dygraph import Linear\n",
    "import random\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\anaconda\\envs\\work\\lib\\site-packages\\ipykernel\\ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
      "  and should_run_async(code)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>order_detail_id</th>\n",
       "      <th>order_id</th>\n",
       "      <th>order_total_num</th>\n",
       "      <th>order_amount</th>\n",
       "      <th>order_total_payment</th>\n",
       "      <th>order_total_discount</th>\n",
       "      <th>order_pay_time</th>\n",
       "      <th>order_status</th>\n",
       "      <th>order_count</th>\n",
       "      <th>is_customer_rate</th>\n",
       "      <th>...</th>\n",
       "      <th>customer_gender</th>\n",
       "      <th>member_status</th>\n",
       "      <th>is_member_actived</th>\n",
       "      <th>goods_id</th>\n",
       "      <th>goods_class_id</th>\n",
       "      <th>goods_price</th>\n",
       "      <th>goods_status</th>\n",
       "      <th>goods_has_discount</th>\n",
       "      <th>goods_list_time</th>\n",
       "      <th>goods_delist_time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1000000</td>\n",
       "      <td>1000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>239.9</td>\n",
       "      <td>96.9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2012-11-01 00:10:56</td>\n",
       "      <td>6</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>998</td>\n",
       "      <td>998</td>\n",
       "      <td>54.909289</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2014-10-25 11:08:07</td>\n",
       "      <td>2014-11-01 11:08:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1001530</td>\n",
       "      <td>1001327</td>\n",
       "      <td>2.0</td>\n",
       "      <td>288.0</td>\n",
       "      <td>96.9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2013-08-31 23:14:42</td>\n",
       "      <td>6</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1953</td>\n",
       "      <td>1953</td>\n",
       "      <td>45.961352</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2013-08-28 17:27:50</td>\n",
       "      <td>2013-09-01 00:38:17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1001531</td>\n",
       "      <td>1001327</td>\n",
       "      <td>2.0</td>\n",
       "      <td>288.0</td>\n",
       "      <td>96.9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2013-08-31 23:14:42</td>\n",
       "      <td>6</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1083</td>\n",
       "      <td>1083</td>\n",
       "      <td>53.035439</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2014-10-29 18:21:05</td>\n",
       "      <td>2014-11-05 18:21:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1001532</td>\n",
       "      <td>1001328</td>\n",
       "      <td>3.0</td>\n",
       "      <td>180.0</td>\n",
       "      <td>89.7</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2013-08-31 22:06:35</td>\n",
       "      <td>6</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1013</td>\n",
       "      <td>1013</td>\n",
       "      <td>46.046917</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2014-10-25 11:00:00</td>\n",
       "      <td>2014-11-01 11:00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1001533</td>\n",
       "      <td>1001329</td>\n",
       "      <td>1.0</td>\n",
       "      <td>159.9</td>\n",
       "      <td>65.9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2013-08-31 21:33:36</td>\n",
       "      <td>6</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1628</td>\n",
       "      <td>1628</td>\n",
       "      <td>50.722161</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2014-10-23 15:35:33</td>\n",
       "      <td>2014-10-30 15:35:33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2306866</th>\n",
       "      <td>3685495</td>\n",
       "      <td>3238358</td>\n",
       "      <td>1.0</td>\n",
       "      <td>199.0</td>\n",
       "      <td>59.9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2013-01-10 19:24:31</td>\n",
       "      <td>6</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1173</td>\n",
       "      <td>1173</td>\n",
       "      <td>53.012016</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2014-10-25 10:14:59</td>\n",
       "      <td>2014-11-01 10:14:59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2306867</th>\n",
       "      <td>3685496</td>\n",
       "      <td>3238359</td>\n",
       "      <td>2.0</td>\n",
       "      <td>299.8</td>\n",
       "      <td>89.9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2013-01-27 15:00:27</td>\n",
       "      <td>6</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2513</td>\n",
       "      <td>2513</td>\n",
       "      <td>42.693822</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2014-01-10 15:56:40</td>\n",
       "      <td>2014-01-11 12:46:19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2306868</th>\n",
       "      <td>3685497</td>\n",
       "      <td>3238359</td>\n",
       "      <td>2.0</td>\n",
       "      <td>299.8</td>\n",
       "      <td>89.9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2013-01-27 15:00:27</td>\n",
       "      <td>6</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>998</td>\n",
       "      <td>998</td>\n",
       "      <td>54.889036</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2014-10-25 11:08:07</td>\n",
       "      <td>2014-11-01 11:08:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2306869</th>\n",
       "      <td>3685498</td>\n",
       "      <td>3238360</td>\n",
       "      <td>1.0</td>\n",
       "      <td>168.0</td>\n",
       "      <td>76.9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2012-11-11 00:10:37</td>\n",
       "      <td>6</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1423</td>\n",
       "      <td>1423</td>\n",
       "      <td>52.078004</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2014-10-30 09:31:53</td>\n",
       "      <td>2014-11-06 09:31:53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2306870</th>\n",
       "      <td>3685499</td>\n",
       "      <td>3238361</td>\n",
       "      <td>1.0</td>\n",
       "      <td>102.0</td>\n",
       "      <td>49.9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2013-07-10 14:22:14</td>\n",
       "      <td>6</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1043</td>\n",
       "      <td>1043</td>\n",
       "      <td>46.114765</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2014-10-28 00:03:12</td>\n",
       "      <td>2014-11-04 00:03:12</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2306871 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         order_detail_id  order_id  order_total_num  order_amount  \\\n",
       "0                1000000   1000000              1.0         239.9   \n",
       "1                1001530   1001327              2.0         288.0   \n",
       "2                1001531   1001327              2.0         288.0   \n",
       "3                1001532   1001328              3.0         180.0   \n",
       "4                1001533   1001329              1.0         159.9   \n",
       "...                  ...       ...              ...           ...   \n",
       "2306866          3685495   3238358              1.0         199.0   \n",
       "2306867          3685496   3238359              2.0         299.8   \n",
       "2306868          3685497   3238359              2.0         299.8   \n",
       "2306869          3685498   3238360              1.0         168.0   \n",
       "2306870          3685499   3238361              1.0         102.0   \n",
       "\n",
       "         order_total_payment  order_total_discount       order_pay_time  \\\n",
       "0                       96.9                   0.0  2012-11-01 00:10:56   \n",
       "1                       96.9                   0.0  2013-08-31 23:14:42   \n",
       "2                       96.9                   0.0  2013-08-31 23:14:42   \n",
       "3                       89.7                   0.0  2013-08-31 22:06:35   \n",
       "4                       65.9                   0.0  2013-08-31 21:33:36   \n",
       "...                      ...                   ...                  ...   \n",
       "2306866                 59.9                   0.0  2013-01-10 19:24:31   \n",
       "2306867                 89.9                   0.0  2013-01-27 15:00:27   \n",
       "2306868                 89.9                   0.0  2013-01-27 15:00:27   \n",
       "2306869                 76.9                   0.0  2012-11-11 00:10:37   \n",
       "2306870                 49.9                   0.0  2013-07-10 14:22:14   \n",
       "\n",
       "         order_status  order_count  is_customer_rate  ...  customer_gender  \\\n",
       "0                   6          1.0               0.0  ...              NaN   \n",
       "1                   6          2.0               0.0  ...              NaN   \n",
       "2                   6          2.0               0.0  ...              NaN   \n",
       "3                   6          1.0               0.0  ...              NaN   \n",
       "4                   6          1.0               0.0  ...              NaN   \n",
       "...               ...          ...               ...  ...              ...   \n",
       "2306866             6          1.0               0.0  ...              0.0   \n",
       "2306867             6          2.0               0.0  ...              0.0   \n",
       "2306868             6          2.0               0.0  ...              0.0   \n",
       "2306869             6          1.0               0.0  ...              0.0   \n",
       "2306870             6          1.0               0.0  ...              0.0   \n",
       "\n",
       "         member_status  is_member_actived  goods_id  goods_class_id  \\\n",
       "0                  NaN                NaN       998             998   \n",
       "1                  NaN                NaN      1953            1953   \n",
       "2                  NaN                NaN      1083            1083   \n",
       "3                  NaN                NaN      1013            1013   \n",
       "4                  NaN                NaN      1628            1628   \n",
       "...                ...                ...       ...             ...   \n",
       "2306866            1.0                1.0      1173            1173   \n",
       "2306867            1.0                1.0      2513            2513   \n",
       "2306868            1.0                1.0       998             998   \n",
       "2306869            1.0                1.0      1423            1423   \n",
       "2306870            1.0                1.0      1043            1043   \n",
       "\n",
       "        goods_price goods_status  goods_has_discount      goods_list_time  \\\n",
       "0         54.909289          1.0                 0.0  2014-10-25 11:08:07   \n",
       "1         45.961352          0.0                 1.0  2013-08-28 17:27:50   \n",
       "2         53.035439          1.0                 0.0  2014-10-29 18:21:05   \n",
       "3         46.046917          1.0                 1.0  2014-10-25 11:00:00   \n",
       "4         50.722161          1.0                 0.0  2014-10-23 15:35:33   \n",
       "...             ...          ...                 ...                  ...   \n",
       "2306866   53.012016          1.0                 0.0  2014-10-25 10:14:59   \n",
       "2306867   42.693822          2.0                 0.0  2014-01-10 15:56:40   \n",
       "2306868   54.889036          1.0                 0.0  2014-10-25 11:08:07   \n",
       "2306869   52.078004          1.0                 0.0  2014-10-30 09:31:53   \n",
       "2306870   46.114765          1.0                 1.0  2014-10-28 00:03:12   \n",
       "\n",
       "           goods_delist_time  \n",
       "0        2014-11-01 11:08:07  \n",
       "1        2013-09-01 00:38:17  \n",
       "2        2014-11-05 18:21:05  \n",
       "3        2014-11-01 11:00:00  \n",
       "4        2014-10-30 15:35:33  \n",
       "...                      ...  \n",
       "2306866  2014-11-01 10:14:59  \n",
       "2306867  2014-01-11 12:46:19  \n",
       "2306868  2014-11-01 11:08:07  \n",
       "2306869  2014-11-06 09:31:53  \n",
       "2306870  2014-11-04 00:03:12  \n",
       "\n",
       "[2306871 rows x 29 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv('train.csv')\n",
    "train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 把测试集的id列作为索引，防止误删\n",
    "test  = pd.read_csv('submission.csv').set_index('customer_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1585986"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "174770"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train['customer_id'][train.order_pay_time>'2013-07-31'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2078390"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train[train.order_pay_time<'2013-07-31'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEGCAYAAAB1iW6ZAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAZYElEQVR4nO3dfXSU5Z3/8fcHglBEsUqwboKGn/KwPEiKkYr1Aa260G7x+IAFbV1Wt7S/Ld2uD231dCscXOpq7W89tlgPKvLr1hWttspxVbSuqFvXh2BdSlCQikqw1WjF1tqKge/+MTc4JJPMhEwSvPJ5ncNh5rqv+7q/A5lPrrnue2YUEZiZ2Ydfn54uwMzMysOBbmaWCAe6mVkiHOhmZolwoJuZJcKBbmaWCAe6lYWkb0lqkLRa0rOSPlGk/1JJZ3bymHWSrt2N/WolhaSpnTn+bhx3tqS/aGPbdyU9n/37/UzSfnnbLpW0QdI6SX+V175E0uuS1rQY6/K8/4cH2jqmpceBbp0maTLw18DEiDgcOAnY1NXHjYj6iPiH3dh1FvBf2d/daTbQVrg+CIzL/v3WA5cCSBoDzATGAlOB6yT1zfZZmrW19N2IODwiaoF7gMvKVL/t4RzoVg4HAW9ExHsAEfFGRLwKIOkySU9LWiNpsSS13FnSEZIekbRK0gpJB2Xt/yBpbTbbXFZgvymS7sluz89mrCslvSipYNBnx59BLlxPljQga6/Jn+lKuljS/Oz2kXkz3u/u6JfNuH+Qt889WU19s1cgayT9StIF2auROuCWbJyP5NcVEQ9ERHN29wmgOrt9KrAsIt6LiI3ABmBSts+jwO9aPsaI+H3e3b0Bv3uwl+jRQG/rJWM7/c/KnuANkv69q+uzkj0ADJO0XtJ1ko7P2/aDiDgyIsYBHyE3k99JUj/g+8CZEXEEsARYmG2+BPh4Nmv9cgl1jAb+ilzgzcvGbuloYGNE/BpYCXymhHFvBr6UzXi3ldC/FqiKiHERMR64OSLuAOqBcyKiNiL+1M7+5wH3Zber2PXVTmPW1i5JCyVtAs7BM/Reo6dn6Esp/JKxFUkjyL0M/WREjAX+sevKso6IiHeAI4A5QBNwm6TZ2eYTJD0p6VfAieSWDvKNAsYBD0p6FvgnPpidriY3o/080Exx/5HNZN8AXgcOLNBnFrBjtr+MIssu2Vr2PhHx31lTKROJF4H/I+n72Tr974vtkHe8b5F7rLeUuk8hEfGtiBiWjTO3M2PZh0ePBnqhl4ySDpV0f/by+zFJo7NNXwQWRcRb2b6vd3O51o6I2BYRKyNiHrkAOSNbzriO3Ox7PHADMKDFrgIasllrbUSMj4hTsm2fARYBE4GnJVUUKeO9vNvbgF36Z2vPZwCXSXqJ3CuDqZL2IRei+c+HlnUWUnCf7Gd0ArlXAF8GbixhLLJfgn9Nbha/Y5lkMzAsr1t11laqW8g9ZusFenqGXshi4KvZy++LyQUCwEhgpKRfSHqiu69QsLZJGpW9gtqhFniZD0LxDUmDgEJXtawDKrMTq0jqJ2mspD7AsIh4GPgmMBgY1MlSPwWsjohhEVETEYcAdwKnAa8BQyUdIKk/2dJQRGwB/pB31c7MvPFeAmol9ZE0jGxtW9IQoE9E3EnuFcfErP8fgH0KFZb9PH8DmB4R7+ZtWg7MlNRf0nBgBPBUew+yxf/FqcDz7fW3dBSb8XSr7El/NPCTvHNn/bO/K8j9ME8hN0t5VNL47AlnPWsQ8P1seaKZ3Im7ORGxRdINwBrgt8DTLXeMiK3ZCcNrJQ0m9/98DbkrPX6ctQm4tgz/17OAn7VouxP4vxHxI0kLyIXlZnYNwfOBGyRtBx4B3s7afwFsBNYCzwHPZO1VwM3ZLyXIrlght8R4vaQ/AZNbrKP/gNzP+oPZz/4TEfHliGiQdHt2jGbgKxGxDUDSreSeD0MkNQLzIuIm4F8kjQK2k/vFWsr5B0uAevrjcyXVAPdExDhJ+wLrIuKgAv2uB56MiJuz+w8Bl0REq5AwKydJg7LzBEi6BDgoIr7Ww2WZtbJHLblkl1ttlDQDcpeYSZqQbb6L3Gxkx0vakeROPpl1tc9klxquAY4F/rmnCzIrpEdn6PkvGcmtYc4D/hP4Iblrm/uRuwZ3QXb98PfIXRWzDVgYEa2uTTYz6616fMnFzMzKY49acjEzs93XY1e5DBkyJGpqanrq8GZmH0qrVq16IyIqC23rsUCvqamhvr6+pw5vZvahJOnltrb1qiWX8847j6FDhzJu3LhW2773ve8hiTfeeAOA559/nsmTJ9O/f3+uvvrqksb5+te/zujRozn88MM57bTT2LJlCwBvvvkmJ5xwAoMGDWLu3F3fhb1161bmzJnDyJEjGT16NHfeeWcZH7GZ9Sa9KtBnz57N/fff36p906ZNPPDAAxx88ME72/bff3+uvfZaLr744pLHOfnkk1mzZg2rV69m5MiRXHHFFQAMGDCAyy+/vNUvBoCFCxcydOhQ1q9fz9q1azn++ONb9TEzK0WvCvTjjjuO/fffv1X7BRdcwFVXXUXeu1MZOnQoRx55JP36tf7AvrbGOeWUU6ioyK1iHXXUUTQ2NgKw9957c8wxxzBgQOuPB1myZAmXXpp7I2GfPn0YMmTI7j04M+v1elWgF3L33XdTVVXFhAkTinfugCVLljBt2rR2++xYkvn2t7/NxIkTmTFjBq+99lpZ6zCz3qNXB/q7777Ld77zHRYsWFDWcRcuXEhFRQXnnHNOu/2am5tpbGzk6KOP5plnnmHy5MkFl3jMzErRqwP917/+NRs3bmTChAnU1NTQ2NjIxIkT+e1vf7vbYy5dupR77rmHW265ZZclnEIOOOAABg4cyOmnnw7AjBkzeOaZZ9rdx8ysLXvUpy12t/Hjx/P66x98rPqOSyl3dx37/vvv56qrruKRRx5h4MCBRftL4rOf/SwrV67kxBNP5KGHHmLMmDG7dWwzMyKi3T/kvhLsdWBNkX5Hkvt4zzOLjRkRHHHEEdHdZs6cGR/72MeioqIiqqqq4sYbb9xl+yGHHBJNTU0REfGb3/wmqqqqYp999onBgwdHVVVVvP322+2Oc+ihh0Z1dXVMmDAhJkyYEF/60pd2GfujH/1o7L333lFVVRUNDQ0REfHSSy/FscceG+PHj48TTzwxXn755e74pzCzDymgPtrI1aKf5SLpOOAd4EeR+17IQn36kvvW8j8DSyL3/Yntqquri915Y9HKlSs7vE9vNGXKlJ4uwcy6gKRVEVFXaFvRJZeIeDT7zPL2fJXcFwUc2fHyOm7rtu28v317dxzqQ6dfnz7s1bdXnxox67U6vYYuqYrcV3idQJFAlzSH3BcJ7/Imno56f/t23n2/lC9f730G9sOBbtZLleOk6DXANyNie7GrOiJiMbnvDKWurq7Tn9s7dtLRnR0iKQ1PPd7TJZhZDypHoNcBy7IwHwJ8WlJzRNxVhrHNzKxEnQ70iBi+47akpeS+H/Suzo5rZmYdUzTQC32zOLmvhiMiru/S6szMrGSlXOUyq9TBImJ2p6oxM7Pd5sshzMwS4UA3M0uEA93MLBEOdDOzRDjQzcwS4UA3M0uEA93MLBEOdDOzRDjQzcwS4UA3M0uEA93MLBEOdDOzRDjQzcwS4UA3M0uEA93MLBEOdDOzRDjQzcwS4UA3M0uEA93MLBEOdDOzRDjQzcwSUTTQJS2R9LqkNW1sP0fSakm/kvS4pAnlL9PMzIopZYa+FJjazvaNwPERMR64HFhchrrMzKyDKop1iIhHJdW0s/3xvLtPANVlqMvMzDqo3Gvo5wP3tbVR0hxJ9ZLqm5qaynxoM7PerWyBLukEcoH+zbb6RMTiiKiLiLrKyspyHdrMzChhyaUUkg4HbgSmRcSb5RjTzMw6ptMzdEkHAz8FvhAR6ztfkpmZ7Y6iM3RJtwJTgCGSGoF5QD+AiLgeuAw4ALhOEkBzRNR1VcFmZlZYKVe5zCqy/e+AvytbRWZmtlv8TlEzs0Q40M3MEuFANzNLhAPdzCwRDnQzs0Q40M3MEuFANzNLhAPdzCwRDnQzs0Q40M3MEuFANzNLhAPdzCwRDnQzs0Q40M3MEuFANzNLhAPdzCwRDnQzs0Q40M3MEuFANzNLhAPdzCwRRQNd0hJJr0ta08Z2SbpW0gZJqyVNLH+ZZmZWTCkz9KXA1Ha2TwNGZH/mAD/sfFlmZtZRRQM9Ih4FftdOl1OBH0XOE8B+kg4qV4FmZlaacqyhVwGb8u43Zm2tSJojqV5SfVNTUxkObWZmO3TrSdGIWBwRdRFRV1lZ2Z2HNjNLXjkCfTMwLO9+ddZmZmbdqByBvhw4N7va5Sjg7Yj4TRnGNTOzDqgo1kHSrcAUYIikRmAe0A8gIq4H7gU+DWwA3gX+tquKNTOzthUN9IiYVWR7AF8pW0VmZrZb/E5RM7NEONDNzBLhQDczS4QD3cwsEQ50M7NEONDNzBLhQDczS4QD3cwsEQ50M7NEONDNzBLhQDczS4QD3cwsEQ50M7NEONDNzBLhQDczS4QD3cwsEQ50M7NEONDNzBLhQDczS4QD3cwsESUFuqSpktZJ2iDpkgLbD5b0sKRfSlot6dPlL9XMzNpTNNAl9QUWAdOAMcAsSWNadPsn4PaI+DgwE7iu3IWamVn7SpmhTwI2RMSLEbEVWAac2qJPAPtmtwcDr5avRDMzK0UpgV4FbMq735i15ZsPfF5SI3Av8NVCA0maI6leUn1TU9NulGtmZm0p10nRWcDSiKgGPg38m6RWY0fE4oioi4i6ysrKMh3azMygtEDfDAzLu1+dteU7H7gdICL+GxgADClHgWZmVppSAv1pYISk4ZL2InfSc3mLPq8AnwKQ9JfkAt1rKmZm3ahooEdEMzAXWAE8R+5qlgZJCyRNz7pdBHxR0v8AtwKzIyK6qmgzM2utopROEXEvuZOd+W2X5d1eC3yyvKWZmVlH+J2iZmaJcKCbmSXCgW5mlggHuplZIhzoZmaJcKCbmSXCgW5mlggHuplZIhzoZmaJcKCbmSXCgW5mlggHuplZIhzoZmaJcKCbmSXCgW5mlggHuplZIhzoZmaJcKCbmSXCgW5mlggHuplZIhzoZmaJKCnQJU2VtE7SBkmXtNHnLElrJTVI+vfylmlmZsVUFOsgqS+wCDgZaASelrQ8Itbm9RkBXAp8MiLekjS0qwo2M7PCSpmhTwI2RMSLEbEVWAac2qLPF4FFEfEWQES8Xt4yzcysmFICvQrYlHe/MWvLNxIYKekXkp6QNLXQQJLmSKqXVN/U1LR7FZuZWUHlOilaAYwApgCzgBsk7deyU0Qsjoi6iKirrKws06HNzAxKC/TNwLC8+9VZW75GYHlEvB8RG4H15ALezMy6SSmB/jQwQtJwSXsBM4HlLfrcRW52jqQh5JZgXixfmWZmVkzRQI+IZmAusAJ4Drg9IhokLZA0Peu2AnhT0lrgYeDrEfFmVxVtZmatFb1sESAi7gXubdF2Wd7tAC7M/piZWQ/wO0XNzBLhQDczS4QD3cwsEQ50M7NEONDNzBLhQDczS4QD3cwsEQ50M7NEONDNzBLhQDczS4QD3cwsEQ50M7NEONDNzBLhQDczS4QD3cwsEQ50M7NEONDNzBLhQDczS4QD3cwsEQ50M7NElBTokqZKWidpg6RL2ul3hqSQVFe+Es3MrBRFA11SX2ARMA0YA8ySNKZAv32ArwFPlrtIMzMrrpQZ+iRgQ0S8GBFbgWXAqQX6XQ5cCfy5jPWZmVmJSgn0KmBT3v3GrG0nSROBYRHxH+0NJGmOpHpJ9U1NTR0u1szM2tbpk6KS+gD/D7ioWN+IWBwRdRFRV1lZ2dlDm5lZnlICfTMwLO9+dda2wz7AOGClpJeAo4DlPjFqZta9Sgn0p4ERkoZL2guYCSzfsTEi3o6IIRFRExE1wBPA9Iio75KKzcysoKKBHhHNwFxgBfAccHtENEhaIGl6VxdoZmalqSilU0TcC9zbou2yNvpO6XxZZmbWUX6nqJlZIhzoZmaJcKCbmSXCgW5mlggHuplZIhzoZmaJcKCbmSXCgW5mlggHuplZIhzoZmaJcKCbmSXCgW5mlggHuplZIhzoZmaJcKCbmSXCgW5mlggHuplZIhzoZmaJcKCb2R7tvPPOY+jQoYwbN25n2+c+9zlqa2upra2lpqaG2traXfZ55ZVXGDRoEFdffTUA69at29m/traWfffdl2uuuaboWFdccQWHHXYYo0aNYsWKFV39UDutpO8UNTPrKbNnz2bu3Lmce+65O9tuu+22nbcvuugiBg8evMs+F154IdOmTdt5f9SoUTz77LMAbNu2jaqqKk477bR2x1q7di3Lli2joaGBV199lZNOOon169fTt2/fsj/Gcilphi5pqqR1kjZIuqTA9gslrZW0WtJDkg4pf6lm1hsdd9xx7L///gW3RQS33347s2bN2tl21113MXz4cMaOHVtwn4ceeohDDz2UQw7ZNaZajnX33Xczc+ZM+vfvz/DhwznssMN46qmnyvSoukbRQJfUF1gETAPGALMkjWnR7ZdAXUQcDtwBXFXuQs3MWnrsscc48MADGTFiBADvvPMOV155JfPmzWtzn2XLlu3yC6CtsTZv3sywYcN2bq+urmbz5s1lfgTlVcoMfRKwISJejIitwDLg1PwOEfFwRLyb3X0CqC5vmWZmrd166627hPP8+fO54IILGDRoUMH+W7duZfny5cyYMaPoWB9GpayhVwGb8u43Ap9op//5wH2FNkiaA8wBOPjgg0ss0cystebmZn7605+yatWqnW1PPvkkd9xxB9/4xjfYsmULffr0YcCAAcydOxeA++67j4kTJ3LggQcWHauqqopNmz6IvsbGRqqqqrr4UXVOWU+KSvo8UAccX2h7RCwGFgPU1dVFOY9tZr3Lz3/+c0aPHk119QcLAo899tjO2/Pnz2fQoEE7wxzanoUXGmv69OmcffbZXHjhhbz66qu88MILTJo0qYseTXmUsuSyGRiWd786a9uFpJOAbwHTI+K98pRnZr3drFmzmDx5MuvWraO6upqbbroJaHstvC1//OMfefDBBzn99NNbbSs01tixYznrrLMYM2YMU6dOZdGiRXv0FS4Aimh/oiypAlgPfIpckD8NnB0RDXl9Pk7uZOjUiHihlAPX1dVFfX19hwteuXIlf3y/mXff38bYSUd3eP+UNTz1OAP79WXvfhVMmTKlp8uxXm7lypU9XcIeb3eep5JWRURdoW1Fl1wiolnSXGAF0BdYEhENkhYA9RGxHPguMAj4iSSAVyJieocrNbOkbN22nfe3b+/pMvY4/fr0Ya++5X9fZ0lr6BFxL3Bvi7bL8m6fVOa6zCwB72/fzrvvb+vpMvY4A/vRc4FuZtYZXh79QMNTj3fZ2P4sFzOzRDjQzcwS4UA3M0uEA93MLBEOdDOzRDjQzcwS4UA3M0uEA93MLBEOdDOzRDjQzcwS4UA3M0uEA93MLBEOdDOzRDjQzcwS4UA3M0uEA93MLBEOdDOzRDjQzcwS4UA3M0uEA93MLBElBbqkqZLWSdog6ZIC2/tLui3b/qSkmrJXamZm7Soa6JL6AouAacAYYJakMS26nQ+8FRGHAf8KXFnuQs3MrH0VJfSZBGyIiBcBJC0DTgXW5vU5FZif3b4D+IEkRUSUsdZWGp56vCuHN7My8XO1e5QS6FXAprz7jcAn2uoTEc2S3gYOAN7I7yRpDjAnu/uOpHW7U/QeaAgtHquZ7XFSeZ4e0taGUgK9bCJiMbC4O4/ZHSTVR0RdT9dhZm3rDc/TUk6KbgaG5d2vztoK9pFUAQwG3ixHgWZmVppSAv1pYISk4ZL2AmYCy1v0WQ78TXb7TOA/u3r93MzMdlV0ySVbE58LrAD6AksiokHSAqA+IpYDNwH/JmkD8Dtyod+bJLeMZJag5J+n8kTazCwNfqeomVkiHOhmZolwoHeApNmS/qKNbSslJX1JlNmeSlKNpDU9XUdPc6B3zGygYKCbmfW0Xhvo2W/05yTdIKlB0gOSPpJtq5X0hKTVkn4m6aOSzgTqgFskPbujbwtfyLatkTQpG2u+pIvzjrsmO/YCSf+Y175Q0te69lGbJa1C0i3Z8/oOSQMlvSRpCICkuuyVdB9JL0iqzNr7ZB8sWNmz5Xderw30zAhgUUSMBbYAZ2TtPwK+GRGHA78C5kXEHUA9cE5E1EbEnwqMNzAiaoG/B5YUOfYS4FzI/UCRu9Tzx517OGa92ijguoj4S+D35J6HrUTEdnLPtXOyppOA/4mIpm6psgv19kDfGBHPZrdXATWSBgP7RcQjWfv/B44rcbxbASLiUWBfSfu11TEiXgLelPRx4BTglxHhd9ea7b5NEfGL7PaPgWPa6btzQgWcB9zclYV1l279LJc90Ht5t7cBhZZROqLlRf0BNLPrL84BebdvJLcu/zGKz+jNrH3Fnn87n3sRsUnSa5JOJPeJsueQgN4+Q28lIt4G3pJ0bNb0BWDHbP0PwD7t7P45AEnHAG9nY70ETMzaJwLD8/r/DJgKHEnunbhmtvsOljQ5u3028F/knn9HZG1ntOh/I7mZ/E8iYlu3VNjFevsMvS1/A1wvaSDwIvC3WfvSrP1PwOQC6+h/lvRLoB+5l3EAdwLnSmoAngTW7+gcEVslPQxsSeUHyqwHrQO+ImkJue9r+CHwFHCTpMuBlS36Lye31JLEcgv4rf89KjsZ+gwwIyJe6Ol6zHqT7H0j/xoRxxbt/CHhJZcekn2N3wbgIYe5WffKvhv5TuDSnq6lnDxDNzNLhGfoZmaJcKCbmSXCgW5mlggHuplZIhzoZmaJ+F99ejF4xcnnBAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "y = range(1,2)\n",
    " \n",
    "plt.bar(['not buy','buy'], [1585986-174770,174770], alpha=0.5, width=0.3, color='lightblue', edgecolor='grey', lw=3)\n",
    "plt.title('Sales in August 2013', fontsize=10)\n",
    "for a, b in zip(['not buy','buy'], [1585986-174770,174770]):\n",
    "    plt.text(a, b + 0.05, '%.0f' % b, ha='center', va='bottom', fontsize=10)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "训练集使用内容 510.40096282958984 MB\n",
      "测试集使用内存 24.200225830078125 MB\n"
     ]
    }
   ],
   "source": [
    "# 对于特别大的文件，我们需要做一些内存检查\n",
    "mem_train = train.memory_usage(index=True).sum()\n",
    "mem_test=test.memory_usage(index=True).sum()\n",
    "print(u\"训练集使用内容 \"+ str(mem_train/ 1024**2)+\" MB\")\n",
    "print(u\"测试集使用内存 \"+ str(mem_test/ 1024**2)+\" MB\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 内存优化脚本"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# @from: https://www.kaggle.com/arjanso/reducing-dataframe-memory-size-by-65/code\n",
    "# @liscense: Apache 2.0\n",
    "# @author: weijian\n",
    "def reduce_mem_usage(props):\n",
    "    # 计算当前内存\n",
    "    start_mem_usg = props.memory_usage().sum() / 1024 ** 2\n",
    "    print(\"Memory usage of the dataframe is :\", start_mem_usg, \"MB\")\n",
    "    \n",
    "    # 哪些列包含空值，空值用-999填充。why：因为np.nan当做float处理\n",
    "    NAlist = []\n",
    "    for col in props.columns:\n",
    "        # 这里只过滤了objectd格式，如果你的代码中还包含其他类型，请一并过滤\n",
    "        if (props[col].dtypes != object):\n",
    "            \n",
    "            # print(\"**************************\")\n",
    "            # print(\"columns: \", col)\n",
    "            # print(\"dtype before\", props[col].dtype)\n",
    "            \n",
    "            # 判断是否是int类型\n",
    "            isInt = False\n",
    "            mmax = props[col].max()\n",
    "            mmin = props[col].min()\n",
    "            \n",
    "            # Integer does not support NA, therefore Na needs to be filled\n",
    "            if not np.isfinite(props[col]).all():\n",
    "                NAlist.append(col)\n",
    "                props[col].fillna(-999, inplace=True) # 用-999填充\n",
    "                \n",
    "            # test if column can be converted to an integer\n",
    "            asint = props[col].fillna(0).astype(np.int64)\n",
    "            result = np.fabs(props[col] - asint)\n",
    "            result = result.sum()\n",
    "            if result < 0.01: # 绝对误差和小于0.01认为可以转换的，要根据task修改\n",
    "                isInt = True\n",
    "            \n",
    "            # make interger / unsigned Integer datatypes\n",
    "            if isInt:\n",
    "                if mmin >= 0: # 最小值大于0，转换成无符号整型\n",
    "                    if mmax <= 255:\n",
    "                        props[col] = props[col].astype(np.uint8)\n",
    "                    elif mmax <= 65535:\n",
    "                        props[col] = props[col].astype(np.uint16)\n",
    "                    elif mmax <= 4294967295:\n",
    "                        props[col] = props[col].astype(np.uint32)\n",
    "                    else:\n",
    "                        props[col] = props[col].astype(np.uint64)\n",
    "                else: # 转换成有符号整型\n",
    "                    if mmin > np.iinfo(np.int8).min and mmax < np.iinfo(np.int8).max:\n",
    "                        props[col] = props[col].astype(np.int8)\n",
    "                    elif mmin > np.iinfo(np.int16).min and mmax < np.iinfo(np.int16).max:\n",
    "                        props[col] = props[col].astype(np.int16)\n",
    "                    elif mmin > np.iinfo(np.int32).min and mmax < np.iinfo(np.int32).max:\n",
    "                        props[col] = props[col].astype(np.int32)\n",
    "                    elif mmin > np.iinfo(np.int64).min and mmax < np.iinfo(np.int64).max:\n",
    "                        props[col] = props[col].astype(np.int64)  \n",
    "            else: # 注意：这里对于float都转换成float16，需要根据你的情况自己更改\n",
    "                props[col] = props[col].astype(np.float16)\n",
    "            \n",
    "            # print(\"dtype after\", props[col].dtype)\n",
    "            # print(\"********************************\")\n",
    "    print(\"___MEMORY USAGE AFTER COMPLETION:___\")\n",
    "    mem_usg = props.memory_usage().sum() / 1024**2 \n",
    "    print(\"Memory usage is: \",mem_usg,\" MB\")\n",
    "    print(\"This is \",100*mem_usg/start_mem_usg,\"% of the initial size\")\n",
    "    return props, NAlist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "训练集使用内容 349.8006982803345 MB\n",
      "测试集使用内存 24.200225830078125 MB\n"
     ]
    }
   ],
   "source": [
    "# 处理id字段\n",
    "train['order_detail_id'] = train['order_detail_id'].astype(np.uint32)\n",
    "train['order_id'] = train['order_id'].astype(np.uint32)\n",
    "train['customer_id'] = train['customer_id'].astype(np.uint32)\n",
    "train['goods_id'] = train['goods_id'].astype(np.uint32)\n",
    "train['goods_class_id'] = train['goods_class_id'].astype(np.uint32)\n",
    "train['member_id'] = train['member_id'].astype(np.uint32)\n",
    "# 处理状态字段，这里同时处理空值，将空值置为0\n",
    "train['order_status'] = train['order_status'].astype(np.uint8)\n",
    "train['goods_has_discount'] = train['goods_has_discount'].astype(np.uint8)\n",
    "train[\"is_member_actived\"].fillna(0, inplace=True)\n",
    "train[\"is_member_actived\"]=train[\"is_member_actived\"].astype(np.int8)\n",
    "train[\"member_status\"].fillna(0, inplace=True)\n",
    "train[\"member_status\"]=train[\"member_status\"].astype(np.int8)\n",
    "train[\"customer_gender\"].fillna(0, inplace=True)\n",
    "train[\"customer_gender\"]=train[\"customer_gender\"].astype(np.int8)\n",
    "train['is_customer_rate'] = train['is_customer_rate'].astype(np.uint8)\n",
    "train['order_detail_status'] = train['order_detail_status'].astype(np.uint8)\n",
    "# 处理日期\n",
    "train['goods_list_time']=pd.to_datetime(train['goods_list_time'],format=\"%Y-%m-%d\")\n",
    "train['order_pay_time']=pd.to_datetime(train['order_pay_time'],format=\"%Y-%m-%d\")\n",
    "train['goods_delist_time']=pd.to_datetime(train['goods_delist_time'],format=\"%Y-%m-%d\")\n",
    "# 检查内存使用\n",
    "mem_train = train.memory_usage(index=True).sum()\n",
    "mem_test=test.memory_usage(index=True).sum()\n",
    "print(u\"训练集使用内容 \"+ str(mem_train/ 1024**2)+\" MB\")\n",
    "print(u\"测试集使用内存 \"+ str(mem_test/ 1024**2)+\" MB\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 构造时间滑窗特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 将用户下单金额按天进行汇总\n",
    "# df = train[train.order_status<101][train.order_pay_time>'2013-02-01']\n",
    "df = train[train.order_pay_time>'2013-02-01']\n",
    "df['date'] = pd.DatetimeIndex(df['order_pay_time']).date\n",
    "df_payment = df[['customer_id','date','order_total_payment']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "685471"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_payment['customer_id'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_payment = df_payment.groupby(['date','customer_id']).agg({'order_total_payment': ['sum']})\n",
    "df_payment.columns = ['day_total_payment']\n",
    "df_payment.reset_index(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_payment = df_payment.set_index(\n",
    "    [\"customer_id\", \"date\"])[[\"day_total_payment\"]].unstack(level=-1).fillna(0)\n",
    "df_payment.columns = df_payment.columns.get_level_values(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 每日购买商品数量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_goods = df[['customer_id','date','order_total_num']]\n",
    "df_goods = df_goods.groupby(['date','customer_id']).agg({'order_total_num': ['sum']})\n",
    "df_goods.columns = ['day_total_num']\n",
    "df_goods.reset_index(inplace=True)\n",
    "df_goods = df_goods.set_index(\n",
    "    [\"customer_id\", \"date\"])[[\"day_total_num\"]].unstack(level=-1).fillna(0)\n",
    "df_goods.columns = df_goods.columns.get_level_values(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 这是一个时间滑窗函数，获得dt之前minus天以来periods的dataframe，以便进一步计算\n",
    "def get_timespan(df, dt, minus, periods, freq='D'):\n",
    "    return df[pd.date_range(dt - timedelta(days=minus), periods=periods, freq=freq)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. 构造dataset这里有个取巧的地方，因为要预测的9月份除了开学季以外不是非常特殊的月份，因此主要考虑近期的因素，数据集的开始时间也是2月1日，尽量避免了双十一、元旦假期的影响，当然春节假期继续保留。同时，构造数据集的时候保留了customer_id，主要为了与其它特征做整合。\n",
    "2. 通过一个函数整合付款金额和商品数量的时间滑窗，主要是因为分开做到时候合并占用内存更大，并且函数最后在返回值处做了内存优化，用时间代价尽可能避免内存溢出。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_dataset(df_payment, df_goods, t2018, is_train=True):\n",
    "    X = {}\n",
    "    # 整合用户id\n",
    "    tmp = df_payment.reset_index()\n",
    "    X['customer_id'] = tmp['customer_id']\n",
    "    # 消费特征\n",
    "    print('Preparing payment feature...')\n",
    "    for i in [14,30,60,91]:\n",
    "        tmp = get_timespan(df_payment, t2018, i, i)\n",
    "        # X['diff_%s_mean' % i] = tmp_1.diff(axis=1).mean(axis=1).values\n",
    "        X['mean_%s_decay' % i] = (tmp * np.power(0.9, np.arange(i)[::-1])).sum(axis=1).values\n",
    "        # X['mean_%s' % i] = tmp_1.mean(axis=1).values\n",
    "        # X['median_%s' % i] = tmp.median(axis=1).values\n",
    "        # X['min_%s' % i] = tmp_1.min(axis=1).values\n",
    "        X['max_%s' % i] = tmp.max(axis=1).values\n",
    "        # X['std_%s' % i] = tmp_1.std(axis=1).values\n",
    "        X['sum_%s' % i] = tmp.sum(axis=1).values\n",
    "    for i in [14,30,60,91]:\n",
    "        tmp = get_timespan(df_payment, t2018 + timedelta(days=-7), i, i)\n",
    "        X['mean_%s_decay_2' % i] = (tmp * np.power(0.9, np.arange(i)[::-1])).sum(axis=1).values\n",
    "        # X['mean_%s_2' % i] = tmp_2.mean(axis=1).values\n",
    "        # X['median_%s_2' % i] = tmp.median(axis=1).values\n",
    "        # X['min_%s_2' % i] = tmp_2.min(axis=1).values\n",
    "        X['max_%s_2' % i] = tmp.max(axis=1).values\n",
    "        # X['std_%s_2' % i] = tmp_2.std(axis=1).values\n",
    "    for i in [14,30,60,91]:\n",
    "        tmp = get_timespan(df_payment, t2018, i, i)\n",
    "        X['has_sales_days_in_last_%s' % i] = (tmp != 0).sum(axis=1).values\n",
    "        X['last_has_sales_day_in_last_%s' % i] = i - ((tmp != 0) * np.arange(i)).max(axis=1).values\n",
    "        X['first_has_sales_day_in_last_%s' % i] = ((tmp != 0) * np.arange(i, 0, -1)).max(axis=1).values\n",
    "\n",
    "    # 对此处进行微调，主要考虑近期因素\n",
    "    for i in range(1, 4):\n",
    "        X['day_%s_2018' % i] = get_timespan(df_payment, t2018, i*30, 30).sum(axis=1).values\n",
    "    # 商品数量特征，这里故意把时间和消费特征错开，提高时间滑窗的覆盖面\n",
    "    print('Preparing num feature...')\n",
    "    for i in [21,49,84]:\n",
    "            tmp = get_timespan(df_goods, t2018, i, i)\n",
    "            # X['goods_diff_%s_mean' % i] = tmp_1.diff(axis=1).mean(axis=1).values\n",
    "            # X['goods_mean_%s_decay' % i] = (tmp_1 * np.power(0.9, np.arange(i)[::-1])).sum(axis=1).values\n",
    "            X['goods_mean_%s' % i] = tmp.mean(axis=1).values\n",
    "            # X['goods_median_%s' % i] = tmp.median(axis=1).values\n",
    "            # X['goods_min_%s' % i] = tmp_1.min(axis=1).values\n",
    "            X['goods_max_%s' % i] = tmp.max(axis=1).values\n",
    "            # X['goods_std_%s' % i] = tmp_1.std(axis=1).values\n",
    "            X['goods_sum_%s' % i] = tmp.sum(axis=1).values\n",
    "    for i in [21,49,84]:    \n",
    "            tmp = get_timespan(df_goods, t2018 + timedelta(weeks=-1), i, i)\n",
    "            # X['goods_diff_%s_mean_2' % i] = tmp_2.diff(axis=1).mean(axis=1).values\n",
    "            # X['goods_mean_%s_decay_2' % i] = (tmp_2 * np.power(0.9, np.arange(i)[::-1])).sum(axis=1).values\n",
    "            X['goods_mean_%s_2' % i] = tmp.mean(axis=1).values\n",
    "            # X['goods_median_%s_2' % i] = tmp.median(axis=1).values\n",
    "            # X['goods_min_%s_2' % i] = tmp_2.min(axis=1).values\n",
    "            X['goods_max_%s_2' % i] = tmp.max(axis=1).values\n",
    "            X['goods_sum_%s_2' % i] = tmp.sum(axis=1).values\n",
    "    for i in [21,49,84]:    \n",
    "            tmp = get_timespan(df_goods, t2018, i, i)\n",
    "            X['goods_has_sales_days_in_last_%s' % i] = (tmp > 0).sum(axis=1).values\n",
    "            X['goods_last_has_sales_day_in_last_%s' % i] = i - ((tmp > 0) * np.arange(i)).max(axis=1).values\n",
    "            X['goods_first_has_sales_day_in_last_%s' % i] = ((tmp > 0) * np.arange(i, 0, -1)).max(axis=1).values\n",
    "\n",
    "\n",
    "    # 对此处进行微调，主要考虑近期因素\n",
    "    for i in range(1, 4):\n",
    "        X['goods_day_%s_2018' % i] = get_timespan(df_goods, t2018, i*28, 28).sum(axis=1).values\n",
    "\n",
    "    X = pd.DataFrame(X)\n",
    "    \n",
    "    reduce_mem_usage(X)\n",
    "    \n",
    "    if is_train:\n",
    "        # 这样转换之后，打标签直接用numpy切片就可以了\n",
    "        # 当然这里前提是确认付款总额没有负数的问题\n",
    "        X['label'] = df_goods[pd.date_range(t2018, periods=30)].max(axis=1).values\n",
    "        X['label'][X['label'] > 0] = 1\n",
    "        return X\n",
    "    return X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preparing payment feature...\n",
      "Preparing num feature...\n",
      "Memory usage of the dataframe is : 308.55411529541016 MB\n",
      "___MEMORY USAGE AFTER COMPLETION:___\n",
      "Memory usage is:  73.87003993988037  MB\n",
      "This is  23.94070805672356 % of the initial size\n",
      "Preparing payment feature...\n",
      "Preparing num feature...\n",
      "Memory usage of the dataframe is : 308.55411529541016 MB\n",
      "___MEMORY USAGE AFTER COMPLETION:___\n",
      "Memory usage is:  73.87003993988037  MB\n",
      "This is  23.94070805672356 % of the initial size\n",
      "Preparing payment feature...\n",
      "Preparing num feature...\n",
      "Memory usage of the dataframe is : 308.55411529541016 MB\n",
      "___MEMORY USAGE AFTER COMPLETION:___\n",
      "Memory usage is:  73.87003993988037  MB\n",
      "This is  23.94070805672356 % of the initial size\n",
      "Preparing payment feature...\n",
      "Preparing num feature...\n",
      "Memory usage of the dataframe is : 308.55411529541016 MB\n",
      "___MEMORY USAGE AFTER COMPLETION:___\n",
      "Memory usage is:  73.87003993988037  MB\n",
      "This is  23.94070805672356 % of the initial size\n"
     ]
    }
   ],
   "source": [
    "num_days = 4\n",
    "t2017 = date(2013, 7, 1)\n",
    "X_l, y_l = [], []\n",
    "for i in range(num_days):\n",
    "    delta = timedelta(days=7 * i)\n",
    "    # X_tmp, y_tmp = prepare_dataset(df_payment, df_goods, t2017 + delta)\n",
    "    X_tmp = prepare_dataset(df_payment, df_goods, t2017 + delta)\n",
    "    X_tmp = pd.concat([X_tmp], axis=1)\n",
    "\n",
    "    X_l.append(X_tmp)\n",
    "    # y_l.append(y_tmp)\n",
    "\n",
    "X_train = pd.concat(X_l, axis=0)\n",
    "# y_train = np.concatenate(y_l, axis=0)\n",
    "\n",
    "del X_l, y_l"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preparing payment feature...\n",
      "Preparing num feature...\n",
      "Memory usage of the dataframe is : 308.55411529541016 MB\n",
      "___MEMORY USAGE AFTER COMPLETION:___\n",
      "Memory usage is:  73.87003993988037  MB\n",
      "This is  23.94070805672356 % of the initial size\n"
     ]
    }
   ],
   "source": [
    "X_test = prepare_dataset(df_payment, df_goods, date(2013, 9, 1), is_train=False)\n",
    "X_test = pd.concat([X_test], axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 中间结果保存"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train.to_csv('X_train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test.to_csv('X_test.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 训练&推理 - 训练配置以及训练\n",
    "加载特征工程结果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X_train = pd.read_csv('X_train.csv',usecols=['max_30','has_sales_days_in_last_30','first_has_sales_day_in_last_60','goods_sum_49','label'])\n",
    "X_train = pd.read_csv('X_train.csv')\n",
    "X_train.drop(['Unnamed: 0','customer_id'], inplace=True, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "X_val = pd.read_csv('X_val.csv')\n",
    "X_val.drop('Unnamed: 0', inplace=True, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test = pd.read_csv('X_test.csv')\n",
    "# X_test = pd.read_csv('X_test.csv',usecols=['max_30','has_sales_days_in_last_30','first_has_sales_day_in_last_60','goods_sum_49'])\n",
    "X_test.drop(['Unnamed: 0','customer_id'], inplace=True, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean_14_decay</th>\n",
       "      <th>max_14</th>\n",
       "      <th>sum_14</th>\n",
       "      <th>mean_30_decay</th>\n",
       "      <th>max_30</th>\n",
       "      <th>sum_30</th>\n",
       "      <th>mean_60_decay</th>\n",
       "      <th>max_60</th>\n",
       "      <th>sum_60</th>\n",
       "      <th>mean_91_decay</th>\n",
       "      <th>...</th>\n",
       "      <th>goods_has_sales_days_in_last_49</th>\n",
       "      <th>goods_last_has_sales_day_in_last_49</th>\n",
       "      <th>goods_first_has_sales_day_in_last_49</th>\n",
       "      <th>goods_has_sales_days_in_last_84</th>\n",
       "      <th>goods_last_has_sales_day_in_last_84</th>\n",
       "      <th>goods_first_has_sales_day_in_last_84</th>\n",
       "      <th>goods_day_1_2018</th>\n",
       "      <th>goods_day_2_2018</th>\n",
       "      <th>goods_day_3_2018</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>49</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>84</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>49</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>84</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.537</td>\n",
       "      <td>39.9</td>\n",
       "      <td>39.9</td>\n",
       "      <td>3.537</td>\n",
       "      <td>39.9</td>\n",
       "      <td>39.9</td>\n",
       "      <td>3.537</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>49</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>84</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.530</td>\n",
       "      <td>98.9</td>\n",
       "      <td>197.8</td>\n",
       "      <td>2.530</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>37</td>\n",
       "      <td>37</td>\n",
       "      <td>2</td>\n",
       "      <td>37</td>\n",
       "      <td>56</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 66 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   mean_14_decay  max_14  sum_14  mean_30_decay  max_30  sum_30  \\\n",
       "0            0.0     0.0     0.0          0.000     0.0     0.0   \n",
       "1            0.0     0.0     0.0          0.000     0.0     0.0   \n",
       "2            0.0     0.0     0.0          3.537    39.9    39.9   \n",
       "3            0.0     0.0     0.0          0.000     0.0     0.0   \n",
       "4            0.0     0.0     0.0          0.000     0.0     0.0   \n",
       "\n",
       "   mean_60_decay  max_60  sum_60  mean_91_decay  ...  \\\n",
       "0          0.000     0.0     0.0          0.000  ...   \n",
       "1          0.000     0.0     0.0          0.000  ...   \n",
       "2          3.537    39.9    39.9          3.537  ...   \n",
       "3          0.000     0.0     0.0          0.000  ...   \n",
       "4          2.530    98.9   197.8          2.530  ...   \n",
       "\n",
       "   goods_has_sales_days_in_last_49  goods_last_has_sales_day_in_last_49  \\\n",
       "0                                0                                   49   \n",
       "1                                0                                   49   \n",
       "2                                1                                   24   \n",
       "3                                0                                   49   \n",
       "4                                1                                   37   \n",
       "\n",
       "   goods_first_has_sales_day_in_last_49  goods_has_sales_days_in_last_84  \\\n",
       "0                                     0                                0   \n",
       "1                                     0                                0   \n",
       "2                                    24                                1   \n",
       "3                                     0                                0   \n",
       "4                                    37                                2   \n",
       "\n",
       "   goods_last_has_sales_day_in_last_84  goods_first_has_sales_day_in_last_84  \\\n",
       "0                                   84                                     0   \n",
       "1                                   84                                     0   \n",
       "2                                   24                                    24   \n",
       "3                                   84                                     0   \n",
       "4                                   37                                    56   \n",
       "\n",
       "   goods_day_1_2018  goods_day_2_2018  goods_day_3_2018  label  \n",
       "0                 0                 0                 0    0.0  \n",
       "1                 0                 0                 0    0.0  \n",
       "2                 1                 0                 0    0.0  \n",
       "3                 0                 0                 0    1.0  \n",
       "4                 0                 2                 0    1.0  \n",
       "\n",
       "[5 rows x 66 columns]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 选取参与训练的特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['mean_14_decay', 'max_14', 'sum_14', 'mean_30_decay', 'max_30',\n",
      "       'sum_30', 'mean_60_decay', 'max_60', 'sum_60', 'mean_91_decay',\n",
      "       'max_91', 'sum_91', 'mean_14_decay_2', 'max_14_2', 'mean_30_decay_2',\n",
      "       'max_30_2', 'mean_60_decay_2', 'max_60_2', 'mean_91_decay_2',\n",
      "       'max_91_2', 'has_sales_days_in_last_14',\n",
      "       'last_has_sales_day_in_last_14', 'first_has_sales_day_in_last_14',\n",
      "       'has_sales_days_in_last_30', 'last_has_sales_day_in_last_30',\n",
      "       'first_has_sales_day_in_last_30', 'has_sales_days_in_last_60',\n",
      "       'last_has_sales_day_in_last_60', 'first_has_sales_day_in_last_60',\n",
      "       'has_sales_days_in_last_91', 'last_has_sales_day_in_last_91',\n",
      "       'first_has_sales_day_in_last_91', 'day_1_2018', 'day_2_2018',\n",
      "       'day_3_2018', 'goods_mean_21', 'goods_max_21', 'goods_sum_21',\n",
      "       'goods_mean_49', 'goods_max_49', 'goods_sum_49', 'goods_mean_84',\n",
      "       'goods_max_84', 'goods_sum_84', 'goods_mean_21_2', 'goods_max_21_2',\n",
      "       'goods_sum_21_2', 'goods_mean_49_2', 'goods_max_49_2', 'goods_sum_49_2',\n",
      "       'goods_mean_84_2', 'goods_max_84_2', 'goods_sum_84_2',\n",
      "       'goods_has_sales_days_in_last_21',\n",
      "       'goods_last_has_sales_day_in_last_21',\n",
      "       'goods_first_has_sales_day_in_last_21',\n",
      "       'goods_has_sales_days_in_last_49',\n",
      "       'goods_last_has_sales_day_in_last_49',\n",
      "       'goods_first_has_sales_day_in_last_49',\n",
      "       'goods_has_sales_days_in_last_84',\n",
      "       'goods_last_has_sales_day_in_last_84',\n",
      "       'goods_first_has_sales_day_in_last_84', 'goods_day_1_2018',\n",
      "       'goods_day_2_2018', 'goods_day_3_2018', 'label'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "print(X_train.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = X_train[['has_sales_days_in_last_14',\n",
    "       'last_has_sales_day_in_last_14', 'first_has_sales_day_in_last_14',\n",
    "       'has_sales_days_in_last_30', 'last_has_sales_day_in_last_30',\n",
    "       'first_has_sales_day_in_last_30', 'has_sales_days_in_last_60',\n",
    "       'last_has_sales_day_in_last_60', 'first_has_sales_day_in_last_60',\n",
    "       'has_sales_days_in_last_91', 'last_has_sales_day_in_last_91','goods_mean_21', 'goods_max_21', 'goods_sum_21',\n",
    "       'goods_mean_49', 'goods_max_49', 'goods_sum_49', 'goods_mean_84',\n",
    "       'goods_max_84', 'goods_sum_84', 'goods_mean_21_2', 'goods_max_21_2',\n",
    "       'goods_sum_21_2', 'goods_mean_49_2', 'goods_max_49_2', 'goods_sum_49_2',\n",
    "       'goods_mean_84_2', 'goods_max_84_2', 'goods_sum_84_2',\n",
    "       'goods_has_sales_days_in_last_21',\n",
    "       'goods_last_has_sales_day_in_last_21',\n",
    "       'goods_first_has_sales_day_in_last_21',\n",
    "       'goods_has_sales_days_in_last_49',\n",
    "       'goods_last_has_sales_day_in_last_49',\n",
    "       'goods_first_has_sales_day_in_last_49',\n",
    "       'goods_has_sales_days_in_last_84',\n",
    "       'goods_last_has_sales_day_in_last_84',\n",
    "       'goods_first_has_sales_day_in_last_84', 'goods_day_1_2018',\n",
    "       'goods_day_2_2018', 'goods_day_3_2018','label']]\n",
    "X_test = X_test[['has_sales_days_in_last_14',\n",
    "       'last_has_sales_day_in_last_14', 'first_has_sales_day_in_last_14',\n",
    "       'has_sales_days_in_last_30', 'last_has_sales_day_in_last_30',\n",
    "       'first_has_sales_day_in_last_30', 'has_sales_days_in_last_60',\n",
    "       'last_has_sales_day_in_last_60', 'first_has_sales_day_in_last_60',\n",
    "       'has_sales_days_in_last_91', 'last_has_sales_day_in_last_91','goods_mean_21', 'goods_max_21', 'goods_sum_21',\n",
    "       'goods_mean_49', 'goods_max_49', 'goods_sum_49', 'goods_mean_84',\n",
    "       'goods_max_84', 'goods_sum_84', 'goods_mean_21_2', 'goods_max_21_2',\n",
    "       'goods_sum_21_2', 'goods_mean_49_2', 'goods_max_49_2', 'goods_sum_49_2',\n",
    "       'goods_mean_84_2', 'goods_max_84_2', 'goods_sum_84_2',\n",
    "       'goods_has_sales_days_in_last_21',\n",
    "       'goods_last_has_sales_day_in_last_21',\n",
    "       'goods_first_has_sales_day_in_last_21',\n",
    "       'goods_has_sales_days_in_last_49',\n",
    "       'goods_last_has_sales_day_in_last_49',\n",
    "       'goods_first_has_sales_day_in_last_49',\n",
    "       'goods_has_sales_days_in_last_84',\n",
    "       'goods_last_has_sales_day_in_last_84',\n",
    "       'goods_first_has_sales_day_in_last_84', 'goods_day_1_2018',\n",
    "       'goods_day_2_2018', 'goods_day_3_2018']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = X_train.reset_index(drop=True)\n",
    "X_test = X_test.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 数据归一化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = (X_train - X_train.min()) / (X_train.max() - X_train.min())\n",
    "X_test = (X_test - X_test.min()) / (X_test.max() - X_test.min())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 前面标签也被归一化了，还原\n",
    "X_train['label'][X_train['label'] > 0] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "66\n"
     ]
    }
   ],
   "source": [
    "print(len(X_train.columns))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 数据分割\n",
    "def load_data(df,istrain):\n",
    "    # data = np.fromfile(datafile)\n",
    "    data = df\n",
    "\n",
    "    feature_num = len(data.columns)\n",
    "    # 将原始数据进行Reshape\n",
    "    data = np.array(data)\n",
    "    data = data.reshape([-1, feature_num])\n",
    "    \n",
    "    # 训练集和测试集的划分比例\n",
    "    #ratio = 0.8\n",
    "    if istrain == True:\n",
    "        ratio = 0.8\n",
    "        offset = int(data.shape[0] * ratio)\n",
    "        training_data = data[:offset]\n",
    "        test_data = data[offset:]\n",
    "    else:\n",
    "        training_data = data\n",
    "        test_data = None\n",
    "\n",
    "    return training_data, test_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 加载处理后的数据\n",
    "training_data, test_data = load_data(X_train,True)\n",
    "print('train set done.')\n",
    "\n",
    "pre_data, none = load_data(X_test,False)\n",
    "print('test set done.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 构建多层神经网络\n",
    "class Regressor(fluid.dygraph.Layer):\n",
    "    def __init__(self, name_scope):\n",
    "        super(Regressor, self).__init__(name_scope)\n",
    "        name_scope = self.full_name()\n",
    "        # 定义三层全连接层，输出维度是1，激活函数为relu\n",
    "        self.fc1 = Linear(input_dim=41, output_dim=128, act='relu') # 输入层，input dim 为数据维度大小\n",
    "        self.fc2 = Linear(input_dim=128, output_dim=128, act='relu')\n",
    "        self.fc3 = Linear(input_dim=128, output_dim=1, act='sigmoid')\n",
    "    # 网络的前向计算函数\n",
    "    def forward(self, inputs):\n",
    "        fc1 = self.fc1(inputs)\n",
    "        fc2 = self.fc2(fc1)\n",
    "        x = self.fc3(fc2)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "with fluid.dygraph.guard():\n",
    "    # 声明定义好的线性回归模型\n",
    "    model = Regressor(\"Regressor\")\n",
    "    # 开启模型训练模式\n",
    "    model.train()\n",
    "    # 定义优化算法，这里使用Adam Optimizer\n",
    "    # 学习率设置为0.00001\n",
    "    opt = fluid.optimizer.Adam(learning_rate=0.00001, parameter_list=model.parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 使用类别权重对数据不平衡问题进行处理\n",
    "def wce_loss(pred, label, w=48, epsilon=1e-05): # w 是给到 y=1 类别的权重，越大越重视\n",
    "    label = fluid.layers.clip(label, epsilon, 1-epsilon)\n",
    "    pred = fluid.layers.clip(pred, epsilon, 1-epsilon)\n",
    "\n",
    "    loss = -1 * (w * label * fluid.layers.log(pred) + (1 - label) * fluid.layers.log(1 - pred))\n",
    "    loss = fluid.layers.reduce_mean(loss)\n",
    "    return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 模型训练和保存\n",
    "with dygraph.guard(fluid.CPUPlace()):\n",
    "    EPOCH_NUM = 10   # 设置外层循环次数\n",
    "    BATCH_SIZE = 4096  # 设置batch大小\n",
    "    \n",
    "    # 定义外层循环\n",
    "    for epoch_id in range(EPOCH_NUM):\n",
    "        # 在每轮迭代开始之前，将训练数据的顺序随机的打乱\n",
    "        np.random.shuffle(training_data)\n",
    "        # 将训练数据进行拆分\n",
    "        mini_batches = [training_data[k:k+BATCH_SIZE] for k in range(0, len(training_data), BATCH_SIZE)]\n",
    "        \n",
    "        # 定义内层循环\n",
    "        for iter_id, mini_batch in enumerate(mini_batches):\n",
    "            x = np.array(mini_batch[:, :-1]).astype('float32') # 获得当前批次训练数据\n",
    "            y = np.array(mini_batch[:, -1:]).astype('float32') # 获得当前批次训练标签\n",
    "\n",
    "            # 将numpy数据转为飞桨动态图variable形式\n",
    "            buyer_features = dygraph.to_variable(x)\n",
    "            result = dygraph.to_variable(y)\n",
    "            \n",
    "            # 前向计算\n",
    "            predicts = model(buyer_features)\n",
    "            # loss = fluid.layers.log_loss(predicts, prices)\n",
    "            loss = wce_loss(predicts, result)\n",
    "            avg_loss = fluid.layers.mean(loss)\n",
    "            \n",
    "            logloss = fluid.layers.log_loss(predicts, prices)\n",
    "\n",
    "            if iter_id % 20 == 0:\n",
    "                print(\"epoch: {}, iter: {}, loss is: {}\".format(epoch_id, iter_id, avg_loss.numpy()))\n",
    "                # print(predicts)\n",
    "     \n",
    "            # 反向传播\n",
    "            avg_loss.backward()\n",
    "            # 最小化loss,更新参数\n",
    "            opt.minimize(avg_loss)\n",
    "            # 清除梯度\n",
    "            model.clear_gradients()\n",
    "    # 保存模型\n",
    "    fluid.save_dygraph(model.state_dict(), 'MLP_model')\n",
    "    print(\"模型保存成功，模型参数保存在MLP_model中\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "with dygraph.guard():\n",
    "    # 参数为保存模型参数的文件地址\n",
    "    model_dict, _ = fluid.load_dygraph('MLP_model')\n",
    "    model.load_dict(model_dict)\n",
    "    model.eval()\n",
    "    pre = pre_data.astype('float32')\n",
    "    # 将数据转为动态图的variable格式\n",
    "    pre = dygraph.to_variable(pre)\n",
    "    results = model(pre)\n",
    "\n",
    "    print(\"Inference result is {}\".format(results.numpy()[:5]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(results))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "results.numpy().flatten()[:6]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub = pd.read_csv('X_test.csv', usecols=['customer_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_preds = pd.DataFrame(\n",
    "{    \"customer_id\": sub.customer_id, \n",
    "    \"pred\": results.numpy().flatten()}\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_preds.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub = pd.read_csv('submission.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission = pd.merge(sub, df_preds, on='customer_id', how='left')\n",
    "submission.fillna(0,inplace=True)\n",
    "submission = submission[['customer_id','pred']]\n",
    "submission.rename(columns={'customer_id':'customer_id','pred':'result'}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 将概率值转换为用户是否购买的标签\n",
    "def f(x):\n",
    "    if x <= 0.5:\n",
    "        return 0\n",
    "    if x > 0.5:\n",
    "        return 1\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission['result'] = submission['result'].map(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission.to_csv('submission.csv',index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "work",
   "language": "python",
   "name": "work"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6-final"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}